<?php
require_once "common.php";

_write_html_frame('获取58教育数据');
_Log("开始抓取58的数据", 'note');


$index_url = 'http://cd.58.com/jiaoyu.shtml';

//抓取导航数据
_Log("抓取导航数据", 'note');
$index_content = _request($index_url, 'https://www.baidu.com/s?wd=%E6%95%99%E8%82%B2%E5%9F%B9%E8%AE%AD%2058', 864000000);
if ($index_content) {
    _Log("导航数据抓取成功！", 'success');
    phpQuery::newDocumentHTML($index_content);
    $pq_nav = pq(".banner-cont .sublist dt a");
    $pq_nav_size = $pq_nav->size();
    if ($pq_nav_size) {
        _Log("抓取到{$pq_nav_size}个导航！", 'success');

        $navs = array();
        for ($i = 0; $i < $pq_nav_size; $i++) {
            $navs[] = array(
                'title' => ($pq_nav->eq($i)->text()),
                'url' => ($pq_nav->eq($i)->attr('href'))
            );
        }
        //var_dump($navs);
        foreach ($navs as $nav) {

            $nav_title = $nav['title'];
            $nav_url = $nav['url'];
            if(collected($nav_url)){
                _Log("已经抓取过【{$nav_title}】的数据！", 'error');
                continue;
            }
            //开始抓取
            _Log("开始抓取【{$nav_title}】的数据！", 'note');
            $pn = 1;
            $pn_referer = $index_url;
            for ($i = 1; $i <= $pn; $i++) {
                _Log("开始抓取【{$nav_title}】（第{$i}页）的数据！", 'note');
                $pnurl = $nav_url;
                if ($i != 1) {
                    $pnurl = formatUrl($nav_url . '/pn' . $i);
                }
                $pnurl_parse = parse_url($pnurl);
                $pn_content = _request($pnurl, $pn_referer, 86400);
                phpQuery::newDocumentHTML($pn_content);
                $pq_pager_next = pq('.pager a.next');
                if ($pq_pager_next->size()) {
                    //$pq_pager_next->attr('href');
                    $pn++;
                }

                $lista = pq('#infolist .small-tbimg tr .tdiv a');
                $listSzie = $lista->size();
                if ($listSzie) {
                    _Log("（第{$i}页）的数据获取成功！", 'success');
                    //_Log("$pnurl", 'note');
                    for ($lsI = 0; $lsI < $listSzie; $lsI++) {
                        $lsUrl = $lista->eq($lsI)->attr('href');

                        $lsUrl_parse = parse_url($lsUrl);

                        if ($lsUrl_parse['host'] == $pnurl_parse['host']) {
                            //_Log("$lsUrl", 'note');
                            $detail_content = _request($lsUrl, $pnurl, 86400);
                            phpQuery::newDocumentHTML($detail_content);
                            $linkman = pq('.suUl .su_con:eq(0) a')->eq(0)->text();
                            $phone = pq('.suUl .su_con .l_phone')->text();
                            if ($linkman && $phone) {

                                _Log("详细数据获取成功！", 'success');
                                $topCat = pq('.suUl .cona:eq(0)')->text();
                                $Cat = pq('.suUl .cona:eq(1)')->text();
                                $qqurl = pq('#qqWrapBox a')->attr('href');
                                $company = pq('.userinfo>h2');
                                pq('span,script',$company)->remove();
                                $company = $company->text();
                                $qq = getQQ($qqurl, $pnurl);
                                /*
                                _Log($lsUrl, 'note');
                                _Log($linkman, 'note');
                                _Log($qq, 'note');
                                _Log($phone, 'note');
                                _Log($topCat, 'note');
                                _Log($Cat, 'note');
                                */
                                if(!phoneExists($phone)){
                                    _Log("存储{$linkman}，{$company}，{$phone}！", 'success');
                                    saveItem($linkman,$company,$phone,$qq,$nav_title,$topCat,$Cat);
                                }else{
                                    _Log("数据{$linkman}，{$company}，{$phone}！已经存在", 'error');
                                }
                            }else{
                                _Log("详细数据获取失败！", 'error');
                            }

                        }else{
                            _Log("推广消息无法获取！", 'error');
                        }
                    }
                } else {
                    _Log("（第{$i}页）的数据获取失败！", 'error');
                    $pn = $i;
                }
                $pn_referer = $pnurl;

            }
            _Log("抓取【{$nav_title}】的数据完成！", 'note');

        }
    } else {
        _Log("导航数据不存在！", 'error');
    }
} else {

    _Log("导航数据抓取失败！", 'error');
}


//函数定义区
function getQQ($url, $referer = '')
{
    $content = _request($url, $referer);
//http://sighttp.qq.com/authd?IDKEY=fb41b9dd2b7959b92cc3e1d564cd331a31fc6efc034fcbc1
    preg_match_all('/tuin\s*:\s*\'([1-9][0-9]{4,9})\'/is', $content, $qq);
    if ($qq && isset($qq[1]) && isset($qq[1][0])) {
        return $qq[1][0];
    }
    return '';
}
function saveItem($linkman,$company,$phone,$qq,$title,$topCat,$Cat){
    static $head = false;
    $content = '';
    if(!$head){
        $head = true;
        $content.="联系人\t公司\t电话\tQQ\t方向\t类别\t子类别\n";
    }
    $linkman = iTrime($linkman);
    $phone = iTrime($phone);
    $qq = iTrime($qq);
    $topCat = iTrime($topCat);
    $title = iTrime($title);
    $Cat = iTrime($Cat);
    $company = iTrime($company);
    $content.="{$linkman}\t{$company}\t{$phone}\t{$qq}\t{$title}\t{$topCat}\t{$Cat}\n";
    $fp = @fopen(getSaveFilename(), 'a');
    if($fp){
        $stuatus = @fwrite($fp,$content);
        if($stuatus)
            return true;
    }
    return false;
}

function iTrime($string){
    $string = trim($string);
    return $string;
}

function phoneExists($phone)
{
    $fp = @fopen(getSaveFilename(), 'r');
    if ($fp) {
        $exists = false;
        while ($row = fread($fp,2048) ){
            if(false === stripos($row,$phone)){

            }else{
                $exists = true;
            }
            //_Log($row, 'note');
        }
        @fclose($fp);
        return $exists;
    }
    return false;
}


function getSaveFilename()
{
    $file_path = dirname(__FILE__) . '/data/';
    $file_name = '58edu_' . date('Y-m-d') . '_' . NOW_TIME . '.xls';
    if (!is_dir($file_path))
        mkdir($file_path, 0777, true);
    return $file_path . $file_name;
}

function collected($url){
    $url_md5 = md5(strtolower($url));
    $filename = dirname(__FILE__) . '/data/colledted/'._CacheFileName($url_md5);
    if(is_file($filename)) return true;
    $file_path = dirname($filename);
    if (!is_dir($file_path))
        mkdir($file_path, 0777, true);
    file_put_contents($filename,$url);
    return false;
}
